projects / sentencepiece.git / commit
? search:
summary | shortlog | log | commit | commitdiff | tree
(initial) | patch
Import sentencepiece_0.2.2.orig.tar.gz
authorKentaro Hayashi <kenhys@xdump.org>
Sat, 1 Aug 2026 11:20:13 +0000 (20:20 +0900)
committerKentaro Hayashi <kenhys@xdump.org>
Sat, 1 Aug 2026 11:20:13 +0000 (20:20 +0900)
commit07693258d4b6638be80186e4b3f83a59cf36bccb
treecee77c989813eb52f566135075084df1fc86063atree | snapshot
Import sentencepiece_0.2.2.orig.tar.gz

[dgit import orig sentencepiece_0.2.2.orig.tar.gz]
271 files changed:
.github/ISSUE_TEMPLATE/bug_report.md [new file with mode: 0644] blob
.github/ISSUE_TEMPLATE/feature_request.md [new file with mode: 0644] blob
.github/dependabot.yml [new file with mode: 0644] blob
.github/pull_request_template.md [new file with mode: 0644] blob
.github/workflows/cifuzz.yml [new file with mode: 0644] blob
.github/workflows/cmake.yml [new file with mode: 0644] blob
.github/workflows/cross_build.yml [new file with mode: 0644] blob
.github/workflows/requirements/base.in [new file with mode: 0644] blob
.github/workflows/requirements/base.txt [new file with mode: 0644] blob
.github/workflows/requirements/cibuildwheel.in [new file with mode: 0644] blob
.github/workflows/requirements/cibuildwheel.txt [new file with mode: 0644] blob
.github/workflows/wheel.yml [new file with mode: 0644] blob
.gitignore [new file with mode: 0644] blob
CMakeLists.txt [new file with mode: 0644] blob
CONTRIBUTING.md [new file with mode: 0644] blob
LICENSE [new file with mode: 0644] blob
README.md [new file with mode: 0644] blob
VERSION.txt [new file with mode: 0644] blob
benchmark/.gitignore [new file with mode: 0644] blob
benchmark/README.md [new file with mode: 0644] blob
benchmark/analyze_results.py [new file with mode: 0644] blob
benchmark/download_models.py [new file with mode: 0644] blob
benchmark/prepare_balanced_raw_multilingual.py [new file with mode: 0644] blob
benchmark/run_bench.py [new file with mode: 0644] blob
benchmark/setup_env.sh [new file with mode: 0755] blob
cmake/ios.toolchain.cmake [new file with mode: 0644] blob
cmake/modify_headers.cmake [new file with mode: 0644] blob
config.h.in [new file with mode: 0644] blob
contrib/docker/Dockerfile [new file with mode: 0644] blob
contrib/docker/README.md [new file with mode: 0644] blob
contrib/nlcodec/.gitignore [new file with mode: 0644] blob
contrib/nlcodec/README.md [new file with mode: 0644] blob
contrib/nlcodec/benchmark.sh [new file with mode: 0755] blob
contrib/nlcodec/bpe_model_trainer_nlcodec.cc [new file with mode: 0644] blob
contrib/nlcodec/bpe_model_trainer_nlcodec.h [new file with mode: 0644] blob
contrib/nlcodec/bpe_model_trainer_nlcodec_test.cc [new file with mode: 0644] blob
data/Scripts.txt [new file with mode: 0644] blob
data/botchan.txt [new file with mode: 0644] blob
data/botchan_1000_bpe.model [new file with mode: 0644] blob
data/gen_unicode_scripts_code.pl [new file with mode: 0755] blob
data/ids_denorm.tsv [new file with mode: 0644] blob
data/ids_norm.tsv [new file with mode: 0644] blob
data/korean_hangul_jamo.tsv [new file with mode: 0644] blob
data/nfc.tsv [new file with mode: 0644] blob
data/nfc_cf.tsv [new file with mode: 0644] blob
data/nfd.tsv [new file with mode: 0644] blob
data/nfd_cf.tsv [new file with mode: 0644] blob
data/nfkc.bin [new file with mode: 0644] blob
data/nfkc.tsv [new file with mode: 0644] blob
data/nfkc_cf.bin [new file with mode: 0644] blob
data/nfkc_cf.tsv [new file with mode: 0644] blob
data/nfkd.tsv [new file with mode: 0644] blob
data/nfkd_cf.tsv [new file with mode: 0644] blob
data/nmt_nfkc.bin [new file with mode: 0644] blob
data/nmt_nfkc.tsv [new file with mode: 0644] blob
data/nmt_nfkc_cf.bin [new file with mode: 0644] blob
data/nmt_nfkc_cf.tsv [new file with mode: 0644] blob
data/test_oss_model.model [new file with mode: 0644] blob
data/wagahaiwa_nekodearu.txt [new file with mode: 0644] blob
data/wagahaiwa_nekodearu_2000_bpe_byte.model [new file with mode: 0644] blob
doc/cli.md [new file with mode: 0644] blob
doc/cpp.md [new file with mode: 0644] blob
doc/model_proto.md [new file with mode: 0644] blob
doc/normalization.md [new file with mode: 0644] blob
doc/options.md [new file with mode: 0644] blob
doc/performance_benchmark.md [new file with mode: 0644] blob
doc/piece_constraints.md [new file with mode: 0644] blob
doc/special_symbols.md [new file with mode: 0644] blob
python/.gitignore [new file with mode: 0644] blob
python/MANIFEST.in [new file with mode: 0644] blob
python/README.md [new file with mode: 0644] blob
python/build_bundled.sh [new file with mode: 0755] blob
python/build_sdist.sh [new file with mode: 0755] blob
python/exports.txt [new file with mode: 0644] blob
python/exports_mac.txt [new file with mode: 0644] blob
python/pyproject.toml [new file with mode: 0644] blob
python/setup.cfg [new file with mode: 0644] blob
python/setup.py [new file with mode: 0755] blob
python/src/sentencepiece/__init__.py [new file with mode: 0644] blob
python/src/sentencepiece/__init__.pyi [new file with mode: 0644] blob
python/src/sentencepiece/_version.py [new file with mode: 0644] blob
python/src/sentencepiece/py.typed [new file with mode: 0644] blob
python/src/sentencepiece/sentencepiece_model_pb2.py [new file with mode: 0644] blob
python/src/sentencepiece/sentencepiece_pb2.py [new file with mode: 0644] blob
python/src/sentencepiece/sentencepiece_pybind.cc [new file with mode: 0644] blob
python/test/__init__.py [new file with mode: 0644] blob
python/test/botchan.txt [new file with mode: 0644] blob
python/test/clean_sentencepiece_test_manual.py [new file with mode: 0644] blob
python/test/gen_stubs_test.py [new file with mode: 0644] blob
python/test/gil_release_test.py [new file with mode: 0644] blob
python/test/numpy_test.py [new file with mode: 0644] blob
python/test/sentencepiece_test.py [new file with mode: 0755] blob
python/test/test_bpe_model.model [new file with mode: 0644] blob
python/test/test_ja_model.model [new file with mode: 0644] blob
python/test/test_model.model [new file with mode: 0644] blob
python/test/typing_smoke.py [new file with mode: 0644] blob
python/tokenizer_comparison_cheat_sheet.md [new file with mode: 0644] blob
python/tools/gen_stubs.py [new file with mode: 0644] blob
sentencepiece.pc.in [new file with mode: 0644] blob
src/CMakeLists.txt [new file with mode: 0644] blob
src/bpe_model.cc [new file with mode: 0644] blob
src/bpe_model.h [new file with mode: 0644] blob
src/bpe_model_test.cc [new file with mode: 0644] blob
src/bpe_model_trainer.cc [new file with mode: 0644] blob
src/bpe_model_trainer.h [new file with mode: 0644] blob
src/bpe_model_trainer_test.cc [new file with mode: 0644] blob
src/builder.cc [new file with mode: 0644] blob
src/builder.h [new file with mode: 0644] blob
src/builder_test.cc [new file with mode: 0644] blob
src/builtin_pb/sentencepiece.pb.cc [new file with mode: 0644] blob
src/builtin_pb/sentencepiece.pb.h [new file with mode: 0644] blob
src/builtin_pb/sentencepiece_model.pb.cc [new file with mode: 0644] blob
src/builtin_pb/sentencepiece_model.pb.h [new file with mode: 0644] blob
src/char_model.cc [new file with mode: 0644] blob
src/char_model.h [new file with mode: 0644] blob
src/char_model_test.cc [new file with mode: 0644] blob
src/char_model_trainer.cc [new file with mode: 0644] blob
src/char_model_trainer.h [new file with mode: 0644] blob
src/char_model_trainer_test.cc [new file with mode: 0644] blob
src/common.h [new file with mode: 0644] blob
src/compile_charsmap_main.cc [new file with mode: 0644] blob
src/filesystem.cc [new file with mode: 0644] blob
src/filesystem.h [new file with mode: 0644] blob
src/filesystem_test.cc [new file with mode: 0644] blob
src/freelist.h [new file with mode: 0644] blob
src/freelist_test.cc [new file with mode: 0644] blob
src/init.cc [new file with mode: 0644] blob
src/init.h [new file with mode: 0644] blob
src/invalid_model_test.cc [new file with mode: 0644] blob
src/model_factory.cc [new file with mode: 0644] blob
src/model_factory.h [new file with mode: 0644] blob
src/model_factory_test.cc [new file with mode: 0644] blob
src/model_interface.cc [new file with mode: 0644] blob
src/model_interface.h [new file with mode: 0644] blob
src/model_interface_test.cc [new file with mode: 0644] blob
src/normalization_rule.h [new file with mode: 0644] blob
src/normalizer.cc [new file with mode: 0644] blob
src/normalizer.h [new file with mode: 0644] blob
src/normalizer_test.cc [new file with mode: 0644] blob
src/pretokenizer_for_training.cc [new file with mode: 0644] blob
src/pretokenizer_for_training.h [new file with mode: 0644] blob
src/pretokenizer_for_training_test.cc [new file with mode: 0644] blob
src/sentencepiece.proto [new file with mode: 0644] blob
src/sentencepiece_model.proto [new file with mode: 0644] blob
src/sentencepiece_processor.cc [new file with mode: 0644] blob
src/sentencepiece_processor.h [new file with mode: 0644] blob
src/sentencepiece_processor_benchmarks_test.cc [new file with mode: 0644] blob
src/sentencepiece_processor_parallel_test.cc [new file with mode: 0644] blob
src/sentencepiece_processor_test.cc [new file with mode: 0644] blob
src/sentencepiece_trainer.cc [new file with mode: 0644] blob
src/sentencepiece_trainer.h [new file with mode: 0644] blob
src/sentencepiece_trainer_test.cc [new file with mode: 0644] blob
src/spec_parser.h [new file with mode: 0644] blob
src/spm_decode_main.cc [new file with mode: 0644] blob
src/spm_encode_main.cc [new file with mode: 0644] blob
src/spm_export_vocab_main.cc [new file with mode: 0644] blob
src/spm_normalize_main.cc [new file with mode: 0644] blob
src/spm_train_main.cc [new file with mode: 0644] blob
src/test_main.cc [new file with mode: 0644] blob
src/testharness.cc [new file with mode: 0644] blob
src/testharness.h [new file with mode: 0644] blob
src/trainer_factory.cc [new file with mode: 0644] blob
src/trainer_factory.h [new file with mode: 0644] blob
src/trainer_factory_test.cc [new file with mode: 0644] blob
src/trainer_interface.cc [new file with mode: 0644] blob
src/trainer_interface.h [new file with mode: 0644] blob
src/trainer_interface_test.cc [new file with mode: 0644] blob
src/unicode_script.cc [new file with mode: 0644] blob
src/unicode_script.h [new file with mode: 0644] blob
src/unicode_script_map.h [new file with mode: 0644] blob
src/unicode_script_test.cc [new file with mode: 0644] blob
src/unigram_model.cc [new file with mode: 0644] blob
src/unigram_model.h [new file with mode: 0644] blob
src/unigram_model_test.cc [new file with mode: 0644] blob
src/unigram_model_trainer.cc [new file with mode: 0644] blob
src/unigram_model_trainer.h [new file with mode: 0644] blob
src/unigram_model_trainer_test.cc [new file with mode: 0644] blob
src/util.cc [new file with mode: 0644] blob
src/util.h [new file with mode: 0644] blob
src/util_test.cc [new file with mode: 0644] blob
src/word_model.cc [new file with mode: 0644] blob
src/word_model.h [new file with mode: 0644] blob
src/word_model_test.cc [new file with mode: 0644] blob
src/word_model_trainer.cc [new file with mode: 0644] blob
src/word_model_trainer.h [new file with mode: 0644] blob
src/word_model_trainer_test.cc [new file with mode: 0644] blob
third_party/CMakeLists.txt [new file with mode: 0644] blob
third_party/darts_clone/LICENSE [new file with mode: 0644] blob
third_party/darts_clone/darts.h [new file with mode: 0644] blob
third_party/esaxx/LICENSE [new file with mode: 0644] blob
third_party/esaxx/esa.hxx [new file with mode: 0644] blob
third_party/esaxx/sais.hxx [new file with mode: 0644] blob
third_party/protobuf-lite/LICENSE [new file with mode: 0644] blob
third_party/protobuf-lite/arena.cc [new file with mode: 0644] blob
third_party/protobuf-lite/arenastring.cc [new file with mode: 0644] blob
third_party/protobuf-lite/bytestream.cc [new file with mode: 0644] blob
third_party/protobuf-lite/coded_stream.cc [new file with mode: 0644] blob
third_party/protobuf-lite/common.cc [new file with mode: 0644] blob
third_party/protobuf-lite/extension_set.cc [new file with mode: 0644] blob
third_party/protobuf-lite/generated_enum_util.cc [new file with mode: 0644] blob
third_party/protobuf-lite/generated_message_table_driven_lite.cc [new file with mode: 0644] blob
third_party/protobuf-lite/generated_message_util.cc [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/any.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/arena.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/arena_impl.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/arenastring.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/descriptor.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/extension_set.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/extension_set_inl.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/generated_enum_reflection.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/generated_enum_util.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/generated_message_table_driven.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/generated_message_table_driven_lite.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/generated_message_util.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/has_bits.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/implicit_weak_message.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/io/coded_stream.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/io/io_win32.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/io/zero_copy_stream.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl_lite.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/map.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/map_entry_lite.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/map_field_lite.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/map_type_handler.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/message_lite.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/metadata_lite.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/parse_context.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/port.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/port_def.inc [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/port_undef.inc [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/repeated_field.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/bytestream.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/callback.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/casts.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/common.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/hash.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/int128.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/logging.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/macros.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/map_util.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/mutex.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/once.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/platform_macros.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/port.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/status.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/statusor.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/stl_util.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/stringpiece.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/stringprintf.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/strutil.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/stubs/time.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/unknown_field_set.h [new file with mode: 0644] blob
third_party/protobuf-lite/google/protobuf/wire_format_lite.h [new file with mode: 0644] blob
third_party/protobuf-lite/implicit_weak_message.cc [new file with mode: 0644] blob
third_party/protobuf-lite/int128.cc [new file with mode: 0644] blob
third_party/protobuf-lite/io_win32.cc [new file with mode: 0644] blob
third_party/protobuf-lite/message_lite.cc [new file with mode: 0644] blob
third_party/protobuf-lite/parse_context.cc [new file with mode: 0644] blob
third_party/protobuf-lite/repeated_field.cc [new file with mode: 0644] blob
third_party/protobuf-lite/status.cc [new file with mode: 0644] blob
third_party/protobuf-lite/statusor.cc [new file with mode: 0644] blob
third_party/protobuf-lite/stringpiece.cc [new file with mode: 0644] blob
third_party/protobuf-lite/stringprintf.cc [new file with mode: 0644] blob
third_party/protobuf-lite/structurally_valid.cc [new file with mode: 0644] blob
third_party/protobuf-lite/strutil.cc [new file with mode: 0644] blob
third_party/protobuf-lite/time.cc [new file with mode: 0644] blob
third_party/protobuf-lite/wire_format_lite.cc [new file with mode: 0644] blob
third_party/protobuf-lite/zero_copy_stream.cc [new file with mode: 0644] blob
third_party/protobuf-lite/zero_copy_stream_impl.cc [new file with mode: 0644] blob
third_party/protobuf-lite/zero_copy_stream_impl_lite.cc [new file with mode: 0644] blob
Unnamed repository; edit this file 'description' to name the repository.